#pragma once
#ifndef __CLENANPAGE_H__
#define __CLENANPAGE_H__
#include <unordered_map>
#include "simhash/Simhasher.hpp"
#include "Configuration.h"
#include "WebPage.h"
#include <fstream>
using std::unordered_map;
using std::pair;
using std::ofstream;
using namespace simhash;

const char* const DICT_PATH = "../conf/dict/jieba.dict.utf8";
const char* const HMM_PATH = "../conf/dict/hmm_model.utf8";
const char* const IDF_PATH = "../conf/dict/idf.utf8";
const char* const STOP_WORD_PATH = "../conf/dict/stop_words.utf8";

class CleanPage
{
    friend class ReDuplicate;
public:
    CleanPage(const string& confPath,const string& cleanedFilename,const string& noDupIndexFilename)
    :_conf(confPath)
    ,_cleanedFilename(cleanedFilename)
    ,_noDupIndexFilename(noDupIndexFilename)
    ,_simhasher(DICT_PATH,HMM_PATH,
                IDF_PATH,STOP_WORD_PATH)
    {
        //为容器预留一些空间,避免频繁申请内存
        _fingerprint.reserve(100);
    }
    void cleanSingleWebPage(const string& filename);
    void storeSinglePaper(const WebPage& webPage,ofstream& ofs);
    void clean();  //清洗文件
    void reDuplicate();
    void storeIndex();
    void start();   
private:
    Configuration _conf;
    string _cleanedFilename;
    string _noDupIndexFilename;
    vector<WebPage> _cleanedWebPage; //存放一个xml文件解析出的文章
    //清洗后的网页偏移库,三个int分别表示docid,文件的pos,文件的length 
    unordered_map<int,pair<int,int>> _cleanedIndex;
    //用于计算指纹
    Simhasher _simhasher;
    //用来存储计算出的指纹,int为docid,uint64_t表示指纹
    vector<pair<int,uint64_t>> _fingerprint; 
};

#endif
